package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"time"
)

var client *http.Client = http.DefaultClient

var header http.Header = make(http.Header, 1)

var tick = time.Tick(100 * time.Millisecond)

func init() {

	header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36")
}

func Fetcher(request *http.Request) ([]byte, error) {

	//加大访问间隔,以防服务端卡主
	<-tick

	//log.Printf("fetchering is : %s \n", request.URL)

	request.Header = header

	response, doErr := client.Do(request)

	defer func() {

		if response != nil && response.Body != nil {

			response.Body.Close()
		}

	}()

	if doErr != nil {

		return nil, doErr
	}

	if response.StatusCode == http.StatusOK {

		//返回编码encoding
		e := determineEncoding(bufio.NewReader(response.Body))

		utf8Reader := transform.NewReader(response.Body, e.NewDecoder())

		return ioutil.ReadAll(utf8Reader)

	} else {

		return nil, fmt.Errorf("response.StatusCode is %d ", response.StatusCode)
	}

}

//返回html编码
func determineEncoding(r *bufio.Reader) encoding.Encoding {

	bytes, err := r.Peek(1024)

	if err != nil {

		return unicode.UTF8
	}

	e, _, _ := charset.DetermineEncoding(bytes, "")

	//log.Printf("charsetName is : %s", name)

	return e
}
